{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建单层LSTM网络对MNIST数据集分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这里的输入x当成28个时间段，每段内容为28个值，使用unstack将原始的输入28×28调整成具有28个元素的list\n",
    "\n",
    "每个元素为1×28的数组。这28个时序一次送入RNN中，如图下图所示：\n",
    "![](https://gitee.com/kkweishe/images/raw/master/ML/2019-8-17_17-25-47.png)\n",
    "\n",
    "由于是批次操作，所以每次都取该批次中所有图片的一行作为一个时间序列输入。\n",
    "\n",
    "理解了这个转换之后，构建网络就变得很容易了，先建立一个包含128个cell的类lstm_cell，然后将变形后的x1放进去生成节点outputs，最后通过全连接生成pred，最后使用softmax进行分类。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "old_v = tf.compat.v1.logging.get_verbosity()\n",
    "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\n",
    "# 导入 MINST 数据集\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\"data/\", one_hot=True)\n",
    "tf.compat.v1.logging.set_verbosity(old_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0817 18:06:34.657757 140362178049856 module_wrapper.py:136] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0817 18:06:34.662876 140362178049856 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0817 18:06:34.663518 140362178049856 deprecation.py:323] From <ipython-input-2-94e8e97bcb25>:12: BasicLSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0817 18:06:34.665515 140362178049856 deprecation.py:323] From <ipython-input-2-94e8e97bcb25>:13: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
      "W0817 18:06:34.685794 140362178049856 deprecation.py:323] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:734: Layer.add_variable (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.add_weight` method instead.\n",
      "W0817 18:06:34.695257 140362178049856 deprecation.py:506] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/ops/rnn_cell_impl.py:738: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0817 18:06:35.048599 140362178049856 deprecation.py:323] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/contrib/layers/python/layers/layers.py:1866: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "W0817 18:06:35.065407 140362178049856 deprecation.py:323] From <ipython-input-2-94e8e97bcb25>:23: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n",
      "W0817 18:06:35.085622 140362178049856 module_wrapper.py:136] From /usr/local/python3/lib/python3.6/site-packages/tensorflow_core/python/util/module_wrapper.py:163: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iter 1280, Minibatch Loss= 2.123669, Training Accuracy= 0.32031\n",
      "Iter 2560, Minibatch Loss= 1.880366, Training Accuracy= 0.36719\n",
      "Iter 3840, Minibatch Loss= 1.604660, Training Accuracy= 0.41406\n",
      "Iter 5120, Minibatch Loss= 1.290977, Training Accuracy= 0.53906\n",
      "Iter 6400, Minibatch Loss= 1.121061, Training Accuracy= 0.59375\n",
      "Iter 7680, Minibatch Loss= 0.952852, Training Accuracy= 0.71875\n",
      "Iter 8960, Minibatch Loss= 0.892485, Training Accuracy= 0.73438\n",
      "Iter 10240, Minibatch Loss= 0.698507, Training Accuracy= 0.75000\n",
      "Iter 11520, Minibatch Loss= 0.692774, Training Accuracy= 0.79688\n",
      "Iter 12800, Minibatch Loss= 0.712652, Training Accuracy= 0.75781\n",
      "Iter 14080, Minibatch Loss= 0.628787, Training Accuracy= 0.77344\n",
      "Iter 15360, Minibatch Loss= 0.480412, Training Accuracy= 0.85938\n",
      "Iter 16640, Minibatch Loss= 0.431347, Training Accuracy= 0.87500\n",
      "Iter 17920, Minibatch Loss= 0.464947, Training Accuracy= 0.87500\n",
      "Iter 19200, Minibatch Loss= 0.450698, Training Accuracy= 0.87500\n",
      "Iter 20480, Minibatch Loss= 0.382798, Training Accuracy= 0.88281\n",
      "Iter 21760, Minibatch Loss= 0.506578, Training Accuracy= 0.85938\n",
      "Iter 23040, Minibatch Loss= 0.380739, Training Accuracy= 0.89062\n",
      "Iter 24320, Minibatch Loss= 0.345643, Training Accuracy= 0.87500\n",
      "Iter 25600, Minibatch Loss= 0.422373, Training Accuracy= 0.85938\n",
      "Iter 26880, Minibatch Loss= 0.332909, Training Accuracy= 0.89062\n",
      "Iter 28160, Minibatch Loss= 0.269029, Training Accuracy= 0.93750\n",
      "Iter 29440, Minibatch Loss= 0.421532, Training Accuracy= 0.87500\n",
      "Iter 30720, Minibatch Loss= 0.232879, Training Accuracy= 0.92188\n",
      "Iter 32000, Minibatch Loss= 0.412540, Training Accuracy= 0.85156\n",
      "Iter 33280, Minibatch Loss= 0.303702, Training Accuracy= 0.90625\n",
      "Iter 34560, Minibatch Loss= 0.260348, Training Accuracy= 0.92969\n",
      "Iter 35840, Minibatch Loss= 0.181875, Training Accuracy= 0.95312\n",
      "Iter 37120, Minibatch Loss= 0.296286, Training Accuracy= 0.89062\n",
      "Iter 38400, Minibatch Loss= 0.123771, Training Accuracy= 0.96094\n",
      "Iter 39680, Minibatch Loss= 0.184759, Training Accuracy= 0.96094\n",
      "Iter 40960, Minibatch Loss= 0.222321, Training Accuracy= 0.96875\n",
      "Iter 42240, Minibatch Loss= 0.251163, Training Accuracy= 0.92188\n",
      "Iter 43520, Minibatch Loss= 0.264645, Training Accuracy= 0.91406\n",
      "Iter 44800, Minibatch Loss= 0.301121, Training Accuracy= 0.91406\n",
      "Iter 46080, Minibatch Loss= 0.218629, Training Accuracy= 0.91406\n",
      "Iter 47360, Minibatch Loss= 0.150714, Training Accuracy= 0.96094\n",
      "Iter 48640, Minibatch Loss= 0.185175, Training Accuracy= 0.95312\n",
      "Iter 49920, Minibatch Loss= 0.201719, Training Accuracy= 0.92969\n",
      "Iter 51200, Minibatch Loss= 0.299931, Training Accuracy= 0.91406\n",
      "Iter 52480, Minibatch Loss= 0.149284, Training Accuracy= 0.96094\n",
      "Iter 53760, Minibatch Loss= 0.168152, Training Accuracy= 0.94531\n",
      "Iter 55040, Minibatch Loss= 0.166098, Training Accuracy= 0.95312\n",
      "Iter 56320, Minibatch Loss= 0.127440, Training Accuracy= 0.96875\n",
      "Iter 57600, Minibatch Loss= 0.229024, Training Accuracy= 0.92969\n",
      "Iter 58880, Minibatch Loss= 0.204111, Training Accuracy= 0.93750\n",
      "Iter 60160, Minibatch Loss= 0.132831, Training Accuracy= 0.95312\n",
      "Iter 61440, Minibatch Loss= 0.247493, Training Accuracy= 0.91406\n",
      "Iter 62720, Minibatch Loss= 0.170920, Training Accuracy= 0.94531\n",
      "Iter 64000, Minibatch Loss= 0.250919, Training Accuracy= 0.94531\n",
      "Iter 65280, Minibatch Loss= 0.256693, Training Accuracy= 0.93750\n",
      "Iter 66560, Minibatch Loss= 0.258490, Training Accuracy= 0.92969\n",
      "Iter 67840, Minibatch Loss= 0.145542, Training Accuracy= 0.96094\n",
      "Iter 69120, Minibatch Loss= 0.080276, Training Accuracy= 0.98438\n",
      "Iter 70400, Minibatch Loss= 0.186373, Training Accuracy= 0.93750\n",
      "Iter 71680, Minibatch Loss= 0.149742, Training Accuracy= 0.97656\n",
      "Iter 72960, Minibatch Loss= 0.123503, Training Accuracy= 0.96094\n",
      "Iter 74240, Minibatch Loss= 0.109718, Training Accuracy= 0.96875\n",
      "Iter 75520, Minibatch Loss= 0.187299, Training Accuracy= 0.93750\n",
      "Iter 76800, Minibatch Loss= 0.116320, Training Accuracy= 0.95312\n",
      "Iter 78080, Minibatch Loss= 0.200649, Training Accuracy= 0.94531\n",
      "Iter 79360, Minibatch Loss= 0.127103, Training Accuracy= 0.96875\n",
      "Iter 80640, Minibatch Loss= 0.106132, Training Accuracy= 0.97656\n",
      "Iter 81920, Minibatch Loss= 0.122763, Training Accuracy= 0.96875\n",
      "Iter 83200, Minibatch Loss= 0.160990, Training Accuracy= 0.94531\n",
      "Iter 84480, Minibatch Loss= 0.173910, Training Accuracy= 0.95312\n",
      "Iter 85760, Minibatch Loss= 0.147762, Training Accuracy= 0.96094\n",
      "Iter 87040, Minibatch Loss= 0.173037, Training Accuracy= 0.93750\n",
      "Iter 88320, Minibatch Loss= 0.061748, Training Accuracy= 0.99219\n",
      "Iter 89600, Minibatch Loss= 0.132250, Training Accuracy= 0.96094\n",
      "Iter 90880, Minibatch Loss= 0.315201, Training Accuracy= 0.91406\n",
      "Iter 92160, Minibatch Loss= 0.115778, Training Accuracy= 0.94531\n",
      "Iter 93440, Minibatch Loss= 0.120537, Training Accuracy= 0.95312\n",
      "Iter 94720, Minibatch Loss= 0.121461, Training Accuracy= 0.96094\n",
      "Iter 96000, Minibatch Loss= 0.122786, Training Accuracy= 0.96094\n",
      "Iter 97280, Minibatch Loss= 0.115688, Training Accuracy= 0.96875\n",
      "Iter 98560, Minibatch Loss= 0.186289, Training Accuracy= 0.93750\n",
      "Iter 99840, Minibatch Loss= 0.182592, Training Accuracy= 0.94531\n",
      " Finished!\n",
      "Testing Accuracy: 0.984375\n"
     ]
    }
   ],
   "source": [
    "n_input = 28    #MNIST data 输入(img shape: 28*28)\n",
    "n_steps = 28    #序列个数\n",
    "n_hidden = 128  #隐藏层个数\n",
    "n_classes = 10  #MNIST 分类个数 (0～9 digits)\n",
    "\n",
    "# 定义占位符\n",
    "x = tf.placeholder('float', [None, n_steps, n_input])\n",
    "y = tf.placeholder('float', [None, n_classes])\n",
    "\n",
    "# 对矩阵进行分解\n",
    "x1 = tf.unstack(x, n_steps, 1)\n",
    "lstm_cell = tf.contrib.rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)\n",
    "outputs, states = tf.contrib.rnn.static_rnn(lstm_cell, x1, dtype=tf.float32)\n",
    "\n",
    "pred = tf.contrib.layers.fully_connected(outputs[-1], n_classes, activation_fn=None)\n",
    "\n",
    "learning_rate = 0.001\n",
    "training_iters = 100000\n",
    "batch_size = 128\n",
    "display_step = 10\n",
    "\n",
    "# 平均交叉熵损失\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n",
    "\n",
    "## 评估模型\n",
    "# tf.argmax(input,axis)根据axis取值的不同返回每行或者每列最大值的索引。\n",
    "# axis = 1: 行\n",
    "# equal，相等的意思。顾名思义，就是判断，x, y 是不是相等\n",
    "# tf.cast  数据类型转换\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "# 启动session\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    step = 1\n",
    "    \n",
    "    while step * batch_size < training_iters:\n",
    "        batch_x, batch_y = mnist.train.next_batch(batch_size)\n",
    "        \n",
    "        # Reshape data to get 28 seq of 28 elements\n",
    "        batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n",
    "        \n",
    "        sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n",
    "        if step % display_step == 0:\n",
    "            # 计算批次数据的准确率\n",
    "            acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n",
    "            loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n",
    "            \n",
    "            print(\"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n",
    "                  \"{:.5f}\".format(acc))\n",
    "            \n",
    "        step += 1\n",
    "    print (\" Finished!\")\n",
    "    \n",
    "    # 计算准确率 for 128 mnist test images\n",
    "    test_len = 128\n",
    "    test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n",
    "    test_label = mnist.test.labels[:test_len]\n",
    "    print (\"Testing Accuracy:\", \\\n",
    "        sess.run(accuracy, feed_dict={x: test_data, y: test_label}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
